/* 
 * Licensed Materials - Property of IBM
 * (c) Copyright IBM Corporation 2010. All Rights Reserved.
 *
 * Note to U.S. Government Users Restricted Rights:  Use,
 * duplication or disclosure restricted by GSA ADP Schedule
 * Contract with IBM Corp.
 *
 * This code is released under the terms of the Eclipse Public License
 * version 1.0 which can be found at the top level of this project or
 * or at http://www.eclipse.org/org/documents/epl-v10.php
 */

package com.ibm.mining;

import java.io.*;
import java.sql.*;
import java.util.*;
import java.util.logging.*;

import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.NullProgressMonitor;

import com.ibm.team.repository.client.ITeamRepository;
import com.ibm.team.repository.client.TeamPlatform;
import com.ibm.team.repository.client.util.IClientLibraryContext;
import com.ibm.team.repository.common.IChangeEvent;
import com.ibm.team.repository.common.IChangeEventHandle;
import com.ibm.team.repository.common.IContributorHandle;
import com.ibm.team.repository.common.IItemHandle;
import com.ibm.team.repository.common.TeamRepositoryException;
import com.ibm.team.repository.common.UUID;
import com.ibm.team.repository.common.model.query.BaseChangeEventQueryModel.ChangeEventQueryModel;
import com.ibm.team.repository.common.model.query.BaseContributorQueryModel.ContributorQueryModel;
import com.ibm.team.repository.common.query.IItemQuery;
import com.ibm.team.repository.common.query.IItemQueryPage;
import com.ibm.team.repository.common.query.ast.IPredicate;
import com.ibm.team.repository.common.service.IQueryService;
import com.ibm.team.scm.client.IWorkspaceManager;
import com.ibm.team.scm.client.SCMPlatform;
import com.ibm.team.scm.common.IBaseline;
import com.ibm.team.scm.common.IComponent;
import com.ibm.team.scm.common.IComponentHandle;
import com.ibm.team.scm.common.IRepositoryProgressMonitor;
import com.ibm.team.scm.common.IScmService;
import com.ibm.team.scm.common.IWorkspaceHandle;
import com.ibm.team.scm.common.dto.IComponentSearchCriteria;
import com.ibm.team.scm.common.internal.query.BaseComponentQueryModel.ComponentQueryModel;
import com.ibm.team.scm.common.internal.query.BaseWorkspaceQueryModel.WorkspaceQueryModel;
import com.ibm.team.workitem.client.*;
import com.ibm.team.workitem.common.internal.model.query.BaseWorkItemQueryModel;
import com.ibm.team.workitem.common.internal.model.query.BaseWorkItemQueryModel.WorkItemQueryModel;
import com.ibm.team.workitem.common.internal.util.ItemQueryIterator;
import com.ibm.team.workitem.common.model.IWorkItem;
import com.ibm.team.workitem.common.model.IWorkItemHandle;
import com.ibm.team.workitem.common.model.ItemProfile;
import com.ibm.team.build.common.model.IBuildRequestHandle;
import com.ibm.team.build.internal.common.model.query.BaseBuildRequestQueryModel.BuildRequestQueryModel;
import com.ibm.team.links.client.ILinkManager;
import com.ibm.team.links.common.ILink;
import com.ibm.team.links.common.ILinkCollection;
import com.ibm.team.links.common.ILinkQueryPage;
import com.ibm.team.links.common.IReference;
import com.ibm.team.links.common.registry.IEndPointDescriptor;
import com.ibm.team.links.common.registry.ILinkType;
import com.ibm.team.links.common.registry.ILinkTypeRegistry;
import com.ibm.team.process.client.*;
import com.ibm.team.process.common.IDevelopmentLineHandle;
import com.ibm.team.process.common.IProjectArea;
import com.ibm.team.process.common.IProjectAreaHandle;
import com.ibm.team.process.common.ITeamAreaHandle;
import com.ibm.team.process.internal.client.ProcessClientService;
import com.ibm.team.process.internal.common.query.BaseDevelopmentLineQueryModel.DevelopmentLineQueryModel;
import com.ibm.team.process.internal.common.query.BaseProjectAreaQueryModel.ProjectAreaQueryModel;
import com.ibm.team.process.internal.common.query.BaseTeamAreaQueryModel.TeamAreaQueryModel;

import com.ibm.mining.sqldata.*;
import com.ibm.mining.model.*;

public class DataMiner {

	ResearchDatabase db;
	JazzDataInserter jdi;
	Logger logger;
	JazzItemHistoryInserter historyInserter;
	
	
	public static void main(String[] args) throws TeamRepositoryException,
			SQLException, ClassNotFoundException, SecurityException, IOException {
		DataMiner dm = new DataMiner();
		dm.mineData();
	}
			
	public void mineData() throws TeamRepositoryException,
	SQLException, ClassNotFoundException, SecurityException, IOException {

		Properties configFile = new Properties();
	    configFile.load(new FileInputStream("dataminer.properties"));
		
		/* set up logging probably want to change this path at some point... */
	    FileHandler logFileHandler = new FileHandler(configFile.getProperty("LoggingFile"), false);
		logFileHandler.setFormatter(new SimpleFormatter());

		logger = Logger.getLogger("com.ibm.mining");
		logger.setUseParentHandlers(false);
		logger.addHandler(logFileHandler);

		StreamHandler outHandler = new StreamHandler(System.out, new SimpleFormatter()) {
			public void publish(LogRecord record) {
				super.publish(record);
				flush();
			}
		};
		logger.addHandler(outHandler);
		logger.setLevel(Level.ALL);

		logger.info("beginning mining...");

		TeamPlatform.startup();
		try {
			IProgressMonitor monitor = new SysoutProgressMonitor();
			logger.info("Connecting to jazzdata database");
			
			ITeamRepository repository = TeamRepositoryLogin.login(
					configFile.getProperty("JazzServerUrl"), 
					configFile.getProperty("JazzUser"), 
					configFile.getProperty("JazzPassword"), monitor);
			logger.info("Done");

			
			MinerServices minerServices = new MinerServices();
			minerServices.setMonitor(monitor);
			minerServices.setRepository(repository);
			minerServices.setLogger(logger);


			logger.info("Connecting to database");
			JazzMiningModel jazzModel = new JazzMiningModel();
			// FIXME: This should not be hard coded to PostgresResearchDatabase
			ResearchDatabase rdb = new PostgresResearchDatabase();
			Connection conn = rdb.connect(configFile.getProperty("JDBCConnectionString"));
			boolean fromScratch = true;
			if (fromScratch) {
				rdb.dropTablesForModel(jazzModel);
			}
			rdb.createTablesForModel(jazzModel);
			minerServices.setResearchDatabase(rdb);
			/* in case there are problems during mining, we want to be using 
			 * transactions so we can pick up later without problems.
			 */
			conn.setAutoCommit(false);
			logger.info("Done");
			
			historyInserter = new JazzItemHistoryInserter();
			historyInserter.setMinerServices(minerServices);
			
			jdi = new JazzDataInserter(minerServices, conn);
			jdi.createPreparedStatements();
			jdi.fillVisitedSets();

			logger.info("mining data...");
			
			//mineAllWorkItems(minerServices);
			mineData(minerServices);

		} catch (Exception e) {
			StringWriter s = new StringWriter();
			//e.fillInStackTrace();
			e.printStackTrace(new PrintWriter(s));
			logger.severe("Uncaught exception during mining " + e + "\n" + s);
		} finally {
			TeamPlatform.shutdown();
			logger.info("done mining...");
		}
	}
	
	
	/* this is just playing around with links to see what is there */
	public void listLinkTypes(MinerServices minerServices) {
	
		logger.info("listing all link types"); 
		Collection<ILinkType> linkTypes = ILinkTypeRegistry.INSTANCE.allEntries();
		
		for (ILinkType linkType : linkTypes) {
			System.out.println("*** LINK TYPE " + linkType.getLinkTypeId() + " ***");
			System.out.println("is constrained : " + linkType.isConstrained());
			System.out.println("is internal : " + linkType.isInternal());
			
			IEndPointDescriptor source = linkType.getSourceEndPointDescriptor();
			IEndPointDescriptor target = linkType.getTargetEndPointDescriptor();
			
			System.out.println("source : "  + 
					source.getDisplayName() + " : " + 
					(source.getReferencedItemType() == null ? "none" :
							source.getReferencedItemType().getName()) );
			System.out.println("target : "  + 
					target.getDisplayName() + " : " + 
					(target.getReferencedItemType() == null ? "none" : 
					target.getReferencedItemType().getName()) );
			
		}
		
		
	}
	
	void mineData(MinerServices minerServices) throws TeamRepositoryException, SQLException {
		ITeamRepository repository = minerServices.getRepository();
		
		Logger logger = minerServices.logger;
		IProcessClientService processClient = (IProcessClientService) repository
				.getClientLibrary(IProcessClientService.class);
		IAuditableClient auditableClient = (IAuditableClient) repository.getClientLibrary(IAuditableClient.class);

		

		
		int added;
		
		/* visit all build requests */
		minerServices.logger.log(Level.ALL, "beginning team areas");
		BuildRequestQueryModel buildRequestModel = BuildRequestQueryModel.ROOT;
		IItemQuery buildRequestQuery = IItemQuery.FACTORY.newInstance(buildRequestModel);

		ItemQueryIterator<IBuildRequestHandle> buildRequestIterator = new ItemQueryIterator<IBuildRequestHandle>(
				auditableClient, buildRequestQuery);
		added = 0;
		while (buildRequestIterator.hasNext(minerServices.getMonitor())) {
			jdi.visitBuildRequest(buildRequestIterator.next(minerServices.getMonitor()));
			minerServices.getResearchDatabase().commit();
			logger.info("added " + added + " build requests");
			added++;
		}
		logger.info("done with " + added + " build requests");
		
	
		logger.info("beginning workitems");
		WorkItemQueryModel workItemModel = WorkItemQueryModel.ROOT;
		IItemQuery workItemQuery = IItemQuery.FACTORY.newInstance(workItemModel);

		/* visit all work items */
		ItemQueryIterator<IWorkItemHandle> workItemIterator = new ItemQueryIterator<IWorkItemHandle>(
				auditableClient, workItemQuery);
		added = 0;
		boolean didGetData;
		while (workItemIterator.hasNext(minerServices.getMonitor())) {
			IWorkItemHandle workItemHandle= workItemIterator.next(minerServices.getMonitor());
			/* normally the visit method checks if something has been visited internally,
			 * but we also want to avoid visiting the item links so we put it all in a block
			 */
			didGetData = jdi.visitWorkItem(workItemHandle);
			logger.info("added " + added + " workitems");
			if (didGetData) {
				logger.info("doing history changes");
				List <JazzItemChange> changes = historyInserter.getItemHistory(
							(IWorkItem)minerServices.getFullState(workItemHandle)
							);
				System.out.println("inserting " + changes.size() + " changes");
				for (JazzItemChange jic : changes) {
					historyInserter.insertChange(jic);
				}
				minerServices.getResearchDatabase().commit();
			}
			added++;	
		}
		logger.info("done with " + added + " workitems");
	
		
		added = 0;
		logger.log(Level.ALL, "beginning components");
		IComponentSearchCriteria csc = IComponentSearchCriteria.FACTORY.newInstance();
		for (IComponentHandle ch : minerServices.getWorkspaceManager().findComponents(csc,
				1000, minerServices.getMonitor())) {
			jdi.visitComponent(ch);
			added++;
			logger.info("added " + added + " components");
			minerServices.getResearchDatabase().commit();
			
		}
		minerServices.logger.info("done with " + added + " components");	
		
		
		/* Visit project areas
		 * this gets a list of all of the project areas, we have to downcast
		 * because I don't know how else to get a ProcessClientService.
		 */

		minerServices.logger.log(Level.ALL, "beginning project areas");
		ProjectAreaQueryModel projectAreaModel = ProjectAreaQueryModel.ROOT;
		IItemQuery projectAreaQuery = IItemQuery.FACTORY.newInstance(projectAreaModel);

		ItemQueryIterator<IProjectAreaHandle> projectAreaIterator = new ItemQueryIterator<IProjectAreaHandle>(
				auditableClient, projectAreaQuery);
		added = 0;
		while (projectAreaIterator.hasNext(minerServices.getMonitor())) {
			jdi.visitProcessArea(projectAreaIterator.next(minerServices.getMonitor()));
			minerServices.getResearchDatabase().commit();
			logger.info("added " + added + " team areas");
			added++;
		}
		logger.info("done with " + added + " team areas");
		

		
		/* visit all team areas */
		minerServices.logger.log(Level.ALL, "beginning team areas");
		TeamAreaQueryModel teamAreaModel = TeamAreaQueryModel.ROOT;
		IItemQuery teamAreaQuery = IItemQuery.FACTORY.newInstance(teamAreaModel);

		ItemQueryIterator<ITeamAreaHandle> teamAreaIterator = new ItemQueryIterator<ITeamAreaHandle>(
				auditableClient, teamAreaQuery);
		added = 0;
		while (teamAreaIterator.hasNext(minerServices.getMonitor())) {
			jdi.visitProcessArea(teamAreaIterator.next(minerServices.getMonitor()));
			minerServices.getResearchDatabase().commit();
			logger.info("added " + added + " team areas");
			added++;
		}
		logger.info("done with " + added + " team areas");
		
		
		/* visit all development lines, this should also visit all iterations
		 * 
		 */
		minerServices.logger.log(Level.ALL, "beginning development lines");
		DevelopmentLineQueryModel DevelopmentLineModel = DevelopmentLineQueryModel.ROOT;
		IItemQuery DevelopmentLineQuery = IItemQuery.FACTORY.newInstance(DevelopmentLineModel);
		ItemQueryIterator<IDevelopmentLineHandle> DevelopmentLineIterator = new ItemQueryIterator<IDevelopmentLineHandle>(
				auditableClient, DevelopmentLineQuery);
		added = 0;
		while (DevelopmentLineIterator.hasNext(minerServices.getMonitor())) {
			jdi.visitDevelopmentLine(DevelopmentLineIterator.next(minerServices.getMonitor()));
			minerServices.getResearchDatabase().commit();
			logger.info("added " + added + " Development Lines");
			added++;
		}
		minerServices.logger.log(Level.ALL, "done with " + added + " development lines");
		
		// we don't want to have to visit all contributors for all kinds of items
		// so visit them ahead of time and put them in the database
		minerServices.logger.log(Level.ALL, "beginning contributors");
		ContributorQueryModel contributorQueryModel = ContributorQueryModel.ROOT;
		IItemQuery contributorQuery = IItemQuery.FACTORY.newInstance(contributorQueryModel);
		ItemQueryIterator<IContributorHandle> contributorIterator = new ItemQueryIterator<IContributorHandle>(
				auditableClient, contributorQuery);
		added = 0;
		while (contributorIterator.hasNext(minerServices.getMonitor())) {
			jdi.visitContributor(contributorIterator.next(minerServices.getMonitor()));
			minerServices.getResearchDatabase().commit();
			logger.info("added " + added + " contributors");
			added++;
		}
		logger.info("done with " + added + " contributors");
		
		// visit all of the workspaces
//		minerServices.logger.log(Level.ALL, "beginning workspaces");
//	    WorkspaceQueryModel workspaceQueryModel = WorkspaceQueryModel.ROOT;
//	    IItemQuery workspaceQuery = IItemQuery.FACTORY.newInstance(workspaceQueryModel);
//	    ItemQueryIterator<IWorkspaceHandle> workspaceIterator = new ItemQueryIterator<IWorkspaceHandle>(auditableClient, workspaceQuery);
//
//	    added = 0;
//	    while (workspaceIterator.hasNext(minerServices.getMonitor())) {
//			jdi.visitWorkspace(workspaceIterator.next(minerServices.getMonitor()));
//			minerServices.getResearchDatabase().commit();
//			logger.info("added " + added + " workspaces");
//			added++;
//		}
//		logger.info("done with " + added + " workspaces");		    	
	}
}
